#!/usr/bin/perl -w
use DBI;
use warnings;
use Encode;
use utf8;
binmode(STDIN,':encoding(utf8)');
binmode(STDOUT,':encoding(utf8)');
binmode(STDERR,':encoding(utf8)');  
use open ":encoding(utf8)";
use LWP::Protocol::https;
use LWP::UserAgent;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use warnings;
use Time::Local;
use Date::Parse;

&insert_news(1);

sub insert_news(){

    my $page= $_[0];
	my $flag=0;
    my $count;
	my $driver="DBI:mysql";
	my $database="news";
	my $host="localhost";
	my $user="root";
	my $password="192510";
	my $db=DBI->connect("$driver:database=$database;host=$host;user=$user;password=$password");
	

	#得到前几天的时间
	my $current_time=$db->prepare("select UNIX_TIMESTAMP(date_sub(curdate(),interval 7 day));");
	$current_time->execute();
	my @current_time2=$current_time->fetchrow_array;
	my $current_time3=$current_time2[0];
	
	my $url="http://www.wooyun.org/bugs/new_unclaimed/page/$page";
	my ($ua, $request , $response , $content);
	$ua = LWP::UserAgent->new;
	my $agentstr=&randomagent;
	$ua->agent('User-Agent'=>$agentstr);
	$request = HTTP::Request->new('GET',$url);
	$response = $ua->request($request);
	if($response->code ne '200'){
	
	}else{
	    $content = $response->content;
		my $html=decode("utf-8",$content);
		
		my @array=split(/<td><a href="/,$html);
		my $array_num=scalar @array;
		
		for(my $i=1;$i<$array_num;$i++){
			#link
			my $href=$array[$i];
			my @href2=split(/">/,$href);
			my $href3=$href2[0];
			my $rel_href="http://www.wooyun.org/".$href3;
			
			#标题
			my $title1=$array[$i];
			my @title2=split(/<\/a>/,$href2[1]);
			my $title=$title2[0];

			#时间
			my $publishtime=$array[$i-1];
			my @publishtime2=split(/<tr>[\r\n\t]*<th>/,$publishtime);
			my $rel_publishtime=$publishtime2[1];
			$rel_publishtime=~s/<\/th>//g;
			
			#作者
			my $author=$array[$i];
			#print $author."\n\n\n\n";
			my @author2=split(/<th><a title=/,$author);
			my $author3=$author2[2];
			my @author4=split(/href=/,$author3);
			my $rel_author=$author4[0];
			$rel_author=~ s/"//g;
			
			
			my ($ua, $request , $response , $content2);
			$ua = LWP::UserAgent->new;
			$request = HTTP::Request->new('GET',$rel_href);
			$response = $ua->request($request);
			$content2 = $response->content;
			my $html2=decode("utf-8",$content2);
			
			#内容(厂商)
			my @changjia=split(/<h3 class=\'wybug_corp\'>/,$html2);
			my @changjia2=split(/<h3 class='wybug_author'>/,$changjia[1]);
			my @changjia3=split(/<a .*>/,$changjia2[0]);
			my $rel_content=$changjia3[1];
			$rel_content=~ s/<\/a>//g;
			$rel_content=~ s/[\r\n\t]//g;
			$rel_content=~ s/<\/h3>//g;
			
			my $rel_content1="相关厂商:$rel_content";
			
			#时间
			# my @publishtime=split(/<h3 class='wybug_date'>提交时间:/,$html2);
			# my @publishtime2=split(/<\/h3>[\r\n\t]*<h3 class='wybug_open_date'>/,$publishtime[1]);
			# my $rel_publishtime=$publishtime2[0];
			
			#其它
			my $web=3;
			my $m=$title.$rel_author;
			Encode::_utf8_off($m);
			my $md5=md5($m);
			
			#再入库之前先看数据库是否已经有重复的数据
			#my $select=$db->prepare("select ifnull(count(*),0) from cf where nid=3 and nmd5='$md5'");
			#$select->execute();
			#while(my @recs=$select->fetchrow_array){
				#$count=$recs[0];
			#}
			  
			#if($count>0){#说明已经有值了 
			   # $flag=1;
			#}
			   
			#if($flag==1){
				#return;#退出当前的方法说明已经重复了
			#}else{
				  
				  my $bj_publishtime=str2time("$rel_publishtime GMT");
				  
				  if($bj_publishtime>=$current_time3){
				  
						my $q_title=$title;
						$q_title=~s/\s//g;
						
						my $q_rel_href=$rel_href;
						$q_rel_href=~s/\s//g;
						
						my $q_rel_author=$rel_author;
						$q_rel_author=~s/\s//g;
						
						#my $cf_count=0;
						
						if($q_title ne"" and $q_rel_href ne"" and $q_rel_author ne""){
						
						  my $sql= "SET NAMES 'UTF8'";
						  my $sth=$db->prepare($sql);#准备
						  $sth = $db->prepare(q{insert ignore into news(title,web,author,publishtime,content,link,nmd5)VALUES(?,?,?,?,?,?,?)}) or die $db->errstr;
						  $sth->execute($title,$web,$rel_author,$rel_publishtime,$rel_content1,$rel_href,$md5) or die $db->errstr;
						  print "$title\n";
						  # if($cf_count==0){
						  
							# #把该条新闻(也就是当前第一新闻)的nmd5值存入到cf表中
							# my $cf_sth=$db->prepare("update cf set nmd5='$md5' where nid=3");#准备
							# $cf_sth->execute();
						  
						  # }
						  
						  #$cf_count++;
						  
						}
						
				  }else{
				  
						 return;
				  }
			
			
			 #}
			 
		}
	
	
	#if($flag==0){#说明一直没有重复的,继续往前查100条
	   sleep(1);
	   $page=$page+1;
	   &insert_news($page);
	#}
	
	}
	$current_time->finish;
	$db->disconnect();
}

#模拟随机浏览器
sub randomagent
{
## FireFox
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1
# Chrome
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1
##Safari
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13
##Sogou
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
##Baidu
# Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7
##QQ
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.7 (KHTML, like Gecko) Chrome/20.0.1099.0 Safari/536.7 QQBrowser/6.14.15138.201
##Maxthon
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12
##LBBROWSER
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER
##TheWorld
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD
	my @agent_conf=('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
	'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
	'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD');
	my $int=int(rand(12));
	return $agent_conf[$int];
}
